#include "PageLibPreprocessor.h"

PageLibPreProcessor::PageLibPreProcessor()
{
}

PageLibPreProcessor::~PageLibPreProcessor()
{
}

void PageLibPreProcessor::cutRedudantPage()
{
    //读取配置文件
    Configuration &conf = Configuration::GetConfiguration();
    conf.ReadConfiguration("conf/myconf.conf");
    set<string> pageLibDict = conf._configuration["PageLibDict"];
    set<string> storeFileName = conf._configuration["StoreFileName"];
    vector<string> allFiles = conf.ReadFilesInDirectory(*pageLibDict.begin());  

    _fingerprint.clear();

    //遍历文件夹内的文件
    for (auto &filename : allFiles)
    {
        cerr << filename << "\n";
        vector<WebPage> items = _wordCutter.split_XML(filename);

        Simhasher simhasher("dict/jieba.dict.utf8",
                            "dict/hmm_model.utf8",
                            "dict/idf.utf8",
                            "dict/stop_words.utf8");

        size_t topN = 5;

        for (auto it = items.begin(); it != items.end(); ++it)
        {
            uint64_t u64 = 0;
            simhasher.make(it->_link, topN, u64);
            _fingerprint.push_back(u64);
        }

        _pageList.push_back(items[0]);

        for (size_t idx1 = 1; idx1 < _fingerprint.size(); ++idx1)
        {
            bool no_repetition = true;
            for (size_t idx2 = 0; idx2 < idx1; ++idx2)
            {
                if (Simhasher::isEqual(_fingerprint[idx1], _fingerprint[idx2], 2))
                {
                    no_repetition = false;
                }
            }

            if (no_repetition)
            {
                _pageList.push_back(items[idx1]);
            }
        }
    }

    _wordCutter.save_to_JSON(_pageList, *storeFileName.begin());
}

void PageLibPreProcessor::bulidInvertIndexMap()
{
}

void PageLibPreProcessor::storeOnDisk()
{
}
